﻿using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Data;
using System.Data.SqlClient;
using System.Diagnostics;
using System.Diagnostics.Eventing.Reader;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Dapper;
using DotnetSpider.DataFlow;
using DotnetSpider.DataFlow.Parser;
using DotnetSpider.DataFlow.Storage;
using DotnetSpider.Http;
using DotnetSpider.Lawyee.Src;
using DotnetSpider.RequestSupplier;
using DotnetSpider.Scheduler.Component;
using DotnetSpider.Selector;
using DotnetSpider.Statistics.Store;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Serializers;
using MySql.Data.MySqlClient;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using RabbitMQ.Client;
using Serilog;
using StringContent = DotnetSpider.Http.StringContent;

namespace DotnetSpider.Sample.samples
{
	public class FaLvFaGuichinacourt : Spider
	{
		public static async Task RunAsync()
		{
			//var builder = Builder.CreateDefaultBuilder<LawSpider>();  
			var builder = Builder.CreateDefaultBuilder<FaLvFaGuichinacourt>(options =>
			{
				// 每秒 1 个请求
				options.Speed = 20;

				////最大深度
				//options.Depth = 1;

				//最大的队列并发数
				options.RequestedQueueCount = 1000 * 10 * 10 * 10 * 1000;

				// 请求超时
				options.RequestTimeout = 10 * 24 * 60 * 600;

				//尝试次数
				options.RetriedTimes = 50000;

				//Scheduler持续时间
				options.EmptySleepTime = 60 * 60 * 24 * 150;



				//爬虫是否使用代理
				//options.UseProxy = true;

				//去除外链
				//options.RemoveOutboundLinks = true; 

			});


			builder.UseSerilog();
			builder.UseQueueDistinctBfsScheduler<HashSetDuplicateRemover>();
			await builder.Build().RunAsync();
		}
		public static SpiderOptions Options_;
		public FaLvFaGuichinacourt(IOptions<SpiderOptions> options,
			SpiderServices services,
			ILogger<Spider> logger, IStatisticsStore _statisticsStore) : base(
			options, services, logger, _statisticsStore)
		{
			Options_ = options.Value;
		}

		protected override async Task InitializeAsync(CancellationToken stoppingToken)
		{
			
			AddDataFlow(new article_subjectdetail_typeParser());
			AddDataFlow(new article_index_idParser());
			AddDataFlow(new law_more_law_type_id());
			AddDataFlow(new detailParser());

			AddDataFlow(new SqlServerStorage());   //SQL Server存储类

			StartAuto();
			


		}
		public void StartAuto()
		{
			try
			{
				StartLFZZ();
				StartFLFG();
				StartDFFG();
				StartSFJS();
				StartZWTY();
				StartZCCK();
				StartAL();

			}
			catch (Exception ex)
			{

				throw ex;
			}

		}
		#region
		/// <summary>
		/// 立法追踪
		/// </summary>
		/// <param name="startDate"></param>
		/// <param name="totalDays"></param>
		/// <returns></returns>
		public async Task<int> StartAL()
		{
			List<Request> listRequest = new List<Request>();
			//指导案例：行政案例  https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNACSBsYGBgYA.shtml
			if (true)
			{
				Request req = new Request($"https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNACSBsYGBgYA.shtml");
				req.SetProperty("Listpage", "1");
				req.SetProperty("LawType", "指导案例->行政案例");
				req.Name = "article/subjectdetail/type";
				listRequest.Add(req);
			}
			//指导案例：民事案例  https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNACSBkYGBgYA.shtml
			if (true)
			{
				Request req = new Request($"https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNACSBkYGBgYA.shtml");
				req.SetProperty("Listpage", "1");
				req.SetProperty("LawType", "指导案例->民事案例");
				req.Name = "article/subjectdetail/type";
				listRequest.Add(req);
			}
			//指导案例：刑事案例  https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNACSIGQAAA.shtml
			if (true)
			{
				Request req = new Request($"https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNACSIGQAAA.shtml");
				req.SetProperty("Listpage", "1");
				req.SetProperty("LawType", "指导案例->刑事案例");
				req.Name = "article/subjectdetail/type";
				listRequest.Add(req);
			}
			//典型案例：行政案例  https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwMjAwNjAwAAA.shtml
			if (true)
			{
				Request req = new Request($"https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwMjAwNjAwAAA.shtml");
				req.SetProperty("Listpage", "1");
				req.SetProperty("LawType", "典型案例->行政案例");
				req.Name = "article/subjectdetail/type";
				listRequest.Add(req);
			}
			//典型案例：民事案例  https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwAiMDAA.shtml
			if (true)
			{
				Request req = new Request($"https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwAiMDAA.shtml");
				req.SetProperty("Listpage", "1");
				req.SetProperty("LawType", "典型案例->民事案例");
				req.Name = "article/subjectdetail/type";
				listRequest.Add(req);
			}
			//典型案例：刑事案例  https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwMgAyDAwA.shtml
			if (true)
			{
				Request req = new Request($"https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwMgAyDAwA.shtml");
				req.SetProperty("Listpage", "1");
				req.SetProperty("LawType", "典型案例->刑事案例");
				req.Name = "article/subjectdetail/type";
				listRequest.Add(req);
			}
			//其他案例：行政案例  https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwBiMDAA.shtml
			if (true)
			{
				Request req = new Request($"https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwBiMDAA.shtml");
				req.SetProperty("Listpage", "1");
				req.SetProperty("LawType", "其他案例->行政案例");
				req.Name = "article/subjectdetail/type";
				listRequest.Add(req);
			}
			//其他案例：民事案例  https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwNjAwMjAwAAA.shtml
			if (true)
			{
				Request req = new Request($"https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwNjAwMjAwAAA.shtml");
				req.SetProperty("Listpage", "1");
				req.SetProperty("LawType", "其他案例->民事案例");
				req.Name = "article/subjectdetail/type";
				listRequest.Add(req);
			}
			//其他案例：刑事案例  https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwNgAyDAwA.shtml
			if (true)
			{
				Request req = new Request($"https://www.chinacourt.org/article/subjectdetail/type/more/id/MzAwNEiqNDAwNgAyDAwA.shtml");
				req.SetProperty("Listpage", "1");
				req.SetProperty("LawType", "其他案例->刑事案例");
				req.Name = "article/subjectdetail/type";
				listRequest.Add(req);
			}

			return await AddRequestsAsync((IEnumerable<Request>)listRequest.ToArray());
		}



		/// <summary>
		/// 立法追踪
		/// </summary>
		/// <param name="startDate"></param>
		/// <param name="totalDays"></param>
		/// <returns></returns>
		public async Task<int> StartLFZZ()
		{
			List<Request> listRequest = new List<Request>();
			// 立法追踪 https://www.chinacourt.org/article/index/id/MzAwNAAhcwMiAAA.shtml
			Request req = new Request($"https://www.chinacourt.org/article/index/id/MzAwNAAhcwMiAAA.shtml");
			req.SetProperty("Listpage", "1");
			req.SetProperty("LawType", "立法追踪");
			req.Name = "article/index/id";
			listRequest.Add(req);
			return await AddRequestsAsync((IEnumerable<Request>)listRequest.ToArray());
		}

		/// <summary>
		/// 法律法规
		/// </summary>
		/// <returns></returns>
		public async Task<int> StartFLFG()
		{
			List<Request> listRequest = new List<Request>();
			// 法律法规 https://www.chinacourt.org/law/more/law_type_id/MzAwNEAFAA.shtml
			Request req = new Request($"https://www.chinacourt.org/law/more/law_type_id/MzAwNEAFAA.shtml");
			req.SetProperty("Listpage", "1");
			req.SetProperty("LawType", "法律法规");
			req.Name = "law/more/law_type_id";
			listRequest.Add(req);
			return await AddRequestsAsync((IEnumerable<Request>)listRequest.ToArray());
		}

		/// <summary>
		/// 地方法规
		/// </summary>
		/// <returns></returns>
		public async Task<int> StartDFFG()
		{
			List<Request> listRequest = new List<Request>();
			// 地方法规 https://www.chinacourt.org/law/more/law_type_id/MzAwMkAFAA.shtml
			Request req = new Request($"https://www.chinacourt.org/law/more/law_type_id/MzAwMkAFAA.shtml");
			req.SetProperty("Listpage", "1");
			req.SetProperty("LawType", "地方法规");
			req.Name = "law/more/law_type_id";
			listRequest.Add(req);
			return await AddRequestsAsync((IEnumerable<Request>)listRequest.ToArray());
		}

		/// <summary>
		/// 司法解释
		/// </summary>
		/// <returns></returns>
		public async Task<int> StartSFJS()
		{
			List<Request> listRequest = new List<Request>();
			// 司法解释 https://www.chinacourt.org/law/more/law_type_id/MzAwM0AFAA.shtml
			Request req = new Request($"https://www.chinacourt.org/law/more/law_type_id/MzAwM0AFAA.shtml");
			req.SetProperty("Listpage", "1");
			req.SetProperty("LawType", "司法解释");
			req.Name = "law/more/law_type_id";
			listRequest.Add(req);
			return await AddRequestsAsync((IEnumerable<Request>)listRequest.ToArray());
		}

		/// <summary>
		/// 中外条约
		/// </summary>
		/// <returns></returns>
		public async Task<int> StartZWTY()
		{
			List<Request> listRequest = new List<Request>();
			// 中外条约 https://www.chinacourt.org/law/more/law_type_id/MzAwNkAFAA.shtml
			Request req = new Request($"https://www.chinacourt.org/law/more/law_type_id/MzAwNkAFAA.shtml");
			req.SetProperty("Listpage", "1");
			req.SetProperty("LawType", "中外条约");
			req.Name = "law/more/law_type_id";
			listRequest.Add(req);
			return await AddRequestsAsync((IEnumerable<Request>)listRequest.ToArray());
		}

		/// <summary>
		/// 政策参考
		/// </summary>
		/// <returns></returns>
		public async Task<int> StartZCCK()
		{
			List<Request> listRequest = new List<Request>();
			// 政策参考 https://www.chinacourt.org/law/more/law_type_id/MzAwMUAFAA.shtml
			Request req = new Request($"https://www.chinacourt.org/law/more/law_type_id/MzAwMUAFAA.shtml");
			req.SetProperty("Listpage", "1");
			req.SetProperty("LawType", "政策参考");
			req.Name = "law/more/law_type_id";
			listRequest.Add(req);
			return await AddRequestsAsync((IEnumerable<Request>)listRequest.ToArray());
		}
		#endregion
		public class SqlServerStorage : StorageBase
		{
			public override async Task InitAsync()
			{
			}


			protected override async Task StoreAsync(DataContext context)
			{
				var typeName = typeof(News).FullName;
				News data = (News)context.GetData(typeName);
				try
				{
					if (data != null)
					{
						if (data.Title.Length > 200)
						{
							data.Title = data.Title.Substring(0, 200);

						}
						using (var conn = new SqlConnection(Options_.StorageConnectionString))
						{
							
							var have = conn.ExecuteScalar($" select count(1) from Law where  LawID =@LawID ", new { LawID = data.LawID });
							if (have.ToString() == "0")
							{
								conn.Execute($"INSERT INTO Law ([LawID], [Title],[Organ],[StartDate],[ImplementDate],[Type],[Valid],[Other],[Path],[LawType]) VALUES (@LawID, @Title,@Organ,@StartDate,@ImplementDate,@Type,@Valid,@Other,@Path,@LawType) ", data);
								foreach (var item in data.newscontents)
								{
									var have1 = conn.ExecuteScalar($" select count(1) from LawDetail where  Orders =@Orders ", new { Orders = item.Orders });
									if (have1.ToString() == "0")
									{
										if (item.PathName.Length > 300)
										{
											item.PathName = item.PathName.Substring(0, 300);
										}
										conn.Execute($"INSERT INTO LawDetail ([ContentId], [LawID],[PathName],[Content],[Orders],[Type]) VALUES (@ContentId, @LawID,@PathName,@Content,@Orders,@Type) ", item);
									}

								}
							}
							
						}
					}
				}
				catch (Exception ex)
				{
					using (var conn = new SqlConnection(Options_.StorageConnectionString))
					{
						conn.Execute($"delete from LawDetail  WHERE LawID = '" + data.LawID + "' ");
						conn.Execute($"delete from Law WHERE LawID = '" + data.LawID + "'");
						if (data != null)
						{
							conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message + ex.StackTrace + context.Request.RequestUri.AbsoluteUri) + ",data:" + data.LawID + data.Title + "','" + DateTime.Now.ToString() + "') ");
						}
						else
						{
							conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message + ex.StackTrace + context.Request.RequestUri.AbsoluteUri) + ",  :SqlServerStorage,StoreAsync','" + DateTime.Now.ToString() + "') ");
						}
					}
					throw;
				}

			}





		}

		protected override (string Id, string Name) GetIdAndName()
		{
			return (Guid.NewGuid().ToString(), "falvfagui");
		}


		protected class article_subjectdetail_typeParser : DataParser
		{
			public article_subjectdetail_typeParser()
			{
				//AddRequiredValidator("search\\.chinalaw\\.gov\\.cn/AdvanceSearchResult");
				AddRequiredValidatorEx("article/subjectdetail/type");

			}


			protected override Task Parse(DataContext context)
			{

				try
				{
					var List = context.Selectable.Select(Selectors.XPath(".//div[@class='more_right']")).SelectList(Selectors.XPath(".//span[@class='left']"));
					foreach (var item in List)
					{
						//int cou = List.ToList().Count();
						///article/detail/2020/07/id/5327839.shtml
						string href = item.Select(Selectors.XPath(".//a/@href"))?.Value;
						if (!string.IsNullOrEmpty(href))
						{

							string LawName = item.Select(Selectors.XPath(".//a"))?.Value;
							var url = string.Empty;
							if (href.ToLower().IndexOf("http") >= 0)
							{
								url = href;// item.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;

							}
							else
							{
								url = "http://" + context.Request.RequestUri.Host + href;// item.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;

							}
							if (!string.IsNullOrWhiteSpace(url))
							{
								string LawType = context.Request.Properties["LawType"]?.Replace(" ", "");
								var request = context.CreateNewRequest(url);
								Uri uri = new Uri(url);
								string LawID = url.Substring(url.ToLower().IndexOf("/id/") + 4, url.ToLower().IndexOf(".shtml") - url.ToLower().IndexOf("/id/") - 4);
								request.SetProperty("LawType", LawType);
								request.SetProperty("LawID", LawID);
								request.SetProperty("LawName", LawName);

								request.Name = "detail";
								context.AddFollowRequests(request);
							}
						}

					}

					string Listpage = context.Request.Properties["Listpage"]?.Replace(" ", "");
					if (Listpage == "1")
					{
						var pagelist = context.Selectable.Select(Selectors.XPath(".//div[@class='paginationControl']")).SelectList(Selectors.XPath(".//a"));
						foreach (var item in pagelist)
						{
							string name = item?.Value;
							if (!string.IsNullOrEmpty(name) && name == "尾页")
							{
								string wyurl = item.Select(Selectors.XPath("/a/@href"))?.Value;
								///article/index/id/MzAwNAAhcwMiAAA/page/272.shtml
								int totalpage = Convert.ToInt32(wyurl.Substring(wyurl.ToLower().IndexOf("page/") + 5, wyurl.ToLower().IndexOf(".") - wyurl.ToLower().IndexOf("page/") - 5));
								//for (int i = 2; i < 3; i++)
								for (int i = 2; i <= totalpage; i++)
								{

									wyurl = wyurl.Substring(0, wyurl.ToLower().IndexOf("page/")) + "page/" + i.ToString() + ".shtml";

									var url = "http://" + context.Request.RequestUri.Host + wyurl;// item.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;

									if (!string.IsNullOrWhiteSpace(url))
									{
										string LawType = context.Request.Properties["LawType"]?.Replace(" ", "");
										var request = context.CreateNewRequest(url);
										Uri uri = new Uri(url);
										request.SetProperty("Listpage", i.ToString());
										request.SetProperty("LawType", LawType);

										request.Name = "article/subjectdetail/type";
										context.AddFollowRequests(request);
									}
								}
								break;
							}
						}
					}


					return Task.CompletedTask;
				}
				catch (Exception ex)
				{
					using (var conn = new SqlConnection(Options_.StorageConnectionString))
					{
						conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message + ex.StackTrace + context.Request.RequestUri.AbsoluteUri) + ",  :article_subjectdetail_typeParser,Parse' ,'" + DateTime.Now.ToString() + "' )");
					}
					return Task.CompletedTask;

				}

			}
		}


		protected class article_index_idParser : DataParser
		{
			public article_index_idParser()
			{
				//AddRequiredValidator("search\\.chinalaw\\.gov\\.cn/AdvanceSearchResult");
				AddRequiredValidatorEx("article/index/id");

			}


			protected override Task Parse(DataContext context)
			{
				var url = string.Empty;

				try
				{
					var List = context.Selectable.Select(Selectors.XPath(".//div[@id='articleList']")).SelectList(Selectors.XPath(".//span[@class='left']"));
					foreach (var item in List)
					{
						int cou = List.ToList().Count();
						///article/detail/2020/07/id/5327839.shtml
						string href = item.Select(Selectors.XPath(".//a/@href"))?.Value;
						if (!string.IsNullOrEmpty(href))
						{

							string LawName = item.Select(Selectors.XPath(".//a"))?.Value;
							if (href.ToLower().IndexOf("http")>=0)
							{
								url = href;// item.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;

							}
							else
							{
								url = "http://" + context.Request.RequestUri.Host + href;// item.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;

							}
							if (!string.IsNullOrWhiteSpace(url))
							{
								string LawType = context.Request.Properties["LawType"]?.Trim();
								var request = context.CreateNewRequest(url);
								Uri uri = new Uri(url);
								string LawID;

								if (url.IndexOf("/id/")<0)
								{
									continue;
								}
								LawID = url.Substring(url.ToLower().IndexOf("/id/") + 4, url.ToLower().IndexOf(".shtml") - url.ToLower().IndexOf("/id/") - 4);
								request.SetProperty("LawType", LawType);
								request.SetProperty("LawID", LawID);
								request.SetProperty("LawName", LawName);

								request.Name = "detail";
								context.AddFollowRequests(request);
							}
						}

					}

					string Listpage = context.Request.Properties["Listpage"]?.Trim();
					if (Listpage == "1")
					{
						var pagelist = context.Selectable.Select(Selectors.XPath(".//div[@class='paginationControl']")).SelectList(Selectors.XPath(".//a"));
						foreach (var item in pagelist)
						{
							string name = item?.Value;
							if (!string.IsNullOrEmpty(name) && name == "尾页")
							{
								string wyurl = item.Select(Selectors.XPath("/a/@href"))?.Value;
								///article/index/id/MzAwNAAhcwMiAAA/page/272.shtml
								int totalpage = Convert.ToInt32(wyurl.Substring(wyurl.ToLower().IndexOf("page/") + 5, wyurl.ToLower().IndexOf(".") - wyurl.ToLower().IndexOf("page/") - 5));
								//for (int i = 2; i < 3; i++)
								for (int i = 2; i <= totalpage; i++)
								{

									wyurl = wyurl.Substring(0, wyurl.ToLower().IndexOf("page/")) + "page/" + i.ToString() + ".shtml";

									url = "http://" + context.Request.RequestUri.Host + wyurl;// item.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;

									if (!string.IsNullOrWhiteSpace(url))
									{
										string LawType = context.Request.Properties["LawType"]?.Trim();
										var request = context.CreateNewRequest(url);
										Uri uri = new Uri(url);
										request.SetProperty("Listpage", i.ToString());
										request.SetProperty("LawType", LawType);

										request.Name = "article/index/id";
										context.AddFollowRequests(request);
									}
								}
								break;
							}
						}
					}
					

					return Task.CompletedTask;
				}
				catch (Exception ex)
				{
					using (var conn = new SqlConnection(Options_.StorageConnectionString))
					{
						conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message+ex.StackTrace + context.Request.RequestUri.AbsoluteUri) + ",  :article_index_idParser,Parse' ,'" + DateTime.Now.ToString() + "' )");
					}
					return Task.CompletedTask;

				}

			}
		}


		protected class law_more_law_type_id : DataParser
		{
			public law_more_law_type_id()
			{
				//AddRequiredValidator("search\\.chinalaw\\.gov\\.cn/AdvanceSearchResult");
				AddRequiredValidatorEx("law/more/law_type_id");

			}


			protected override Task Parse(DataContext context)
			{
				try
				{
					//var newsList = context.Selectable.Select(Selectors.XPath(".//div[@class='law_list']")).SelectList(Selectors.XPath(".//span[@class='left']/a/@href"));
					var newsList = context.Selectable.Select(Selectors.XPath(".//div[@class='law_list']")).SelectList(Selectors.XPath(".//span[@class='left']"));
					foreach (var item in newsList)
					{
						///article/detail/2020/07/id/5327839.shtml
						string LawName = item.Select(Selectors.XPath(".//a"))?.Value;
						string href = item.Select(Selectors.XPath(".//a/@href"))?.Value;
						if (!string.IsNullOrEmpty(href))
						{
							var url = string.Empty;
							if (href.ToLower().IndexOf("http")>=0)
							{
								 url = href;// item.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;

							}
							else
							{
								url = "http://" + context.Request.RequestUri.Host + href;// item.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;

							}
							if (!string.IsNullOrWhiteSpace(url))
							{
								string LawType = context.Request.Properties["LawType"]?.Trim();
								var request = context.CreateNewRequest(url);
								Uri uri = new Uri(url);
								if (url.IndexOf("/id/") < 0)
								{
									continue;
								}
								string LawID = url.Substring(url.ToLower().IndexOf("/id/") + 4, url.ToLower().IndexOf(".shtml") - url.ToLower().IndexOf("/id/") - 4);
								request.SetProperty("LawType", LawType);
								request.SetProperty("LawID", LawID);
								request.SetProperty("LawName", LawName);

								request.Name = "detail";
								context.AddFollowRequests(request);
							}
						}
					}

					string Listpage = context.Request.Properties["Listpage"]?.Trim();
					if (Listpage == "1")
					{
						var pagelist = context.Selectable.Select(Selectors.XPath(".//div[@class='paginationControl']")).SelectList(Selectors.XPath(".//a"));
						foreach (var item in pagelist)
						{
							string name = item?.Value;
							if (!string.IsNullOrEmpty(name) && name == "尾页")
							{
								string wyurl = item.Select(Selectors.XPath("/a/@href"))?.Value;
								///article/index/id/MzAwNAAhcwMiAAA/page/272.shtml
								int totalpage = Convert.ToInt32(wyurl.Substring(wyurl.ToLower().IndexOf("page/") + 5, wyurl.ToLower().IndexOf(".") - wyurl.ToLower().IndexOf("page/") - 5));
								//for (int i = 2; i < 3; i++)
								for (int i = 2; i <= totalpage; i++)
								{
									wyurl = wyurl.Substring(0, wyurl.ToLower().IndexOf("page/")) + "page/" + i.ToString() + ".shtml";

									var url = "http://" + context.Request.RequestUri.Host + wyurl;// item.Select(Selectors.XPath(".//h3[@class='w_lt']/a/@href"))?.Value;
									if (!string.IsNullOrWhiteSpace(url))
									{
										string LawType = context.Request.Properties["LawType"]?.Trim();
										var request = context.CreateNewRequest(url);
										Uri uri = new Uri(url);
										request.SetProperty("Listpage", i.ToString());
										request.SetProperty("LawType", LawType);

										request.Name = "law/more/law_type_id";

										context.AddFollowRequests(request);
									}
								}
								break;
							}
						}
					}
					return Task.CompletedTask;
				}
				catch (Exception ex)
				{
					using (var conn = new SqlConnection(Options_.StorageConnectionString))
					{
						conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message+ex.StackTrace + context.Request.RequestUri.AbsoluteUri) + ",  :law_more_law_type_id,Parse' ,'" + DateTime.Now.ToString() + "' )");
					}
					return Task.CompletedTask;
				}
			}
		}

		protected class detailParser : DataParser
		{
			public detailParser()
			{
				//AddRequiredValidator("search\\.chinalaw\\.gov\\.cn/law/searchTitleDetail");
				AddRequiredValidatorEx("detail");
			}

			protected override Task Parse(DataContext context)
			{
				if (context.Selectable.Value == null)
				{
					return Task.CompletedTask;
				}
				var typeName = typeof(News).FullName;
				News news = new News();
				try
				{
					news.LawID = context.Request.Properties["LawID"]?.Trim();
					news.Title = context.Request.Properties["LawName"]?.Trim();
					//news.Type = context.Request.Properties["LawType"]?.Trim();
					news.Path = context.Request.RequestUri.AbsoluteUri;
					news.LawType = context.Request.Properties["LawType"]?.Trim();

					string path = context.Request.RequestUri.AbsoluteUri;
					if (path.ToLower().IndexOf("article") >= 0)
					{

						var thr = context.Selectable.Select(Selectors.XPath(".//div[@class='detail_thr']"));
						if (thr != null)
						{
							news.StartDate = thr.Select(Selectors.XPath(".//span[@class='time']"))?.Value;
							news.Organ = context.Selectable.Select(Selectors.XPath(".//div[@class='detail_thr']")).Select(Selectors.XPath(".//span[@class='source']"))?.Value;
							if (news.Title.IndexOf("..") > 0)
							{
								news.Title = context.Selectable.Select(Selectors.XPath(".//div[@class='detail_title']"))?.Value.Replace(" ","");
							}
							NewsContent newscontent;
							var contentlist = context.Selectable.Select(Selectors.XPath(".//div[@class='detail_txt']"));
							if (contentlist != null)
							{
								string content = ((DotnetSpider.Selector.HtmlSelectable)contentlist).OuterHtml;
								newscontent = new NewsContent();
								newscontent.Content = content;
								newscontent.LawID = news.LawID;
								newscontent.PathName = news.Title.Replace(" ","");
								newscontent.Orders = news.LawID + "-aaaaa";
								newscontent.Type = context.Request.Properties["LawType"]?.Trim();
								news.newscontents.Add(newscontent);
							}
							context.AddData(typeName, news);
						}
						else
						{
							string Path = context.Request.GetProperty("Path");

							using (var conn = new SqlConnection(Options_.StorageConnectionString))
							{
								conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('参数有误" + CommonHelper.NoHtml(news.Title + context.Request.RequestUri.AbsoluteUri) + ",  :detailParser,Parse' ,'" + DateTime.Now.ToString() + "') ");
							}
						}
					}
					else if (path.ToLower().IndexOf("law") >= 0)
					{
						var STitle = context.Selectable.Select(Selectors.XPath(".//span[@class='STitle']"));
						if (STitle != null)
						{


							var newsShort = STitle.Value.Replace(" ", "");
							if (!string.IsNullOrEmpty(newsShort))
							{
								MatchCollection mc = Regex.Matches(newsShort, "【.*?】");
								List<string> strs = new List<string>();
								for (int i = 0; i < mc.Count; i++)
								{
									strs.Add(mc[i].Groups[0].Value);

								}
								string[] strArray = strs.ToArray();
								string[] c = newsShort.Split(strArray, StringSplitOptions.None);

								if (c.Length >= 8)
								{

									news.Organ = strs[0] + c[1];
									news.ImplementCode = strs[1] + c[2];
									news.StartDate = strs[2] + c[3];
									news.ImplementDate = strs[3] + c[4];
									news.ValidDate = strs[4] + c[5];
									news.Type = strs[5] + c[6];
									news.Other = strs[6] + c[7];
								}
							}

							NewsContent newscontent;
							var contentlist = context.Selectable.Select(Selectors.XPath(".//div[@class='content_text']"));
							if (news.Title.IndexOf("..") > 0)
							{
								if (contentlist.SelectList(Selectors.XPath(".//strong")) != null && contentlist.SelectList(Selectors.XPath(".//strong")).Count() > 1)
								{
									var tt = contentlist.SelectList(Selectors.XPath(".//strong")).ToList();
									for (int i = 0; i < tt.Count; i++)
									{
										string tming = tt[i]?.Value.Replace(" ", "");
										if (tming != null && tming.IndexOf(news.Title.Substring(0, 6)) >= 0)
										{
											if (tt.Count > i + 2)
											{
												news.Title = tming + tt[i + 1]?.Value.Replace(" ", "") + tt[i + 2]?.Value.Replace(" ", "");

											}
											else if (tt.Count > i + 1)
											{
												news.Title = tming + tt[i + 1]?.Value.Replace(" ", "");

											}
											else
											{
												news.Title = tming;
											}
											break;
										}
									}
								}
								else if (contentlist.SelectList(Selectors.XPath(".//p")) != null && contentlist.SelectList(Selectors.XPath(".//p")).Count() > 2)
								{
									var tt = contentlist.SelectList(Selectors.XPath(".//p")).ToList();

									for (int i = 0; i < tt.Count; i++)
									{
										string tming = tt[i]?.Value.Replace(" ", "");
										if (tming != null && tming.IndexOf(news.Title.Substring(0, 6)) >= 0)
										{
											if (tt.Count > i + 2)
											{
												news.Title = tming + tt[i + 1]?.Value.Replace(" ", "") + tt[i + 2]?.Value.Replace(" ", "");

											}
											else if (tt.Count > i + 1)
											{
												news.Title = tming + tt[i + 1]?.Value.Replace(" ", "");

											}
											else
											{
												news.Title = tming;
											}
											break;
										}
									}
								}
								else
								{
									var tt = contentlist?.Value.Replace(" ", "");
									if (!string.IsNullOrEmpty(tt))
									{
										if (tt.IndexOf(news.Title.Substring(0, 6)) >= 0 && tt.Substring(tt.IndexOf(news.Title.Substring(0, 6))).Length > 50)
										{
											news.Title = tt.Substring(tt.IndexOf(news.Title.Substring(0, 6)), 50);
										}
									}
								}
								
							}
							if (contentlist != null)
							{
								string a1 = "	";
								string b1 = "	";
								string c1 = "　";

								string content = contentlist.Value.Replace(a1, c1).Replace(b1, c1).Replace(" ", c1);

								//Regex reg = new Regex("第.*?条 ");
								MatchCollection mc = Regex.Matches(content, "第.{1,6}?条" + c1);

								List<string> strs = new List<string>();
								for (int i = 0; i < mc.Count; i++)
								{
									if (mc[i].Groups[0].Value.Length <= 8)
									{
										strs.Add(mc[i].Groups[0].Value);
									}
								}

								if (strs.Count < 1)
								{
									newscontent = new NewsContent();
									newscontent.Content = ((DotnetSpider.Selector.HtmlSelectable)contentlist).OuterHtml;
									newscontent.LawID = news.LawID;
									newscontent.PathName = news.Title;
									newscontent.Orders = news.LawID + "-aaaaa";
									newscontent.Type = context.Request.Properties["LawType"]?.Trim();
									news.newscontents.Add(newscontent);
								}
								else
								{

									string[] strArray = strs.ToArray();
									string[] c = content.Split(strArray, StringSplitOptions.RemoveEmptyEntries);

									if (strArray.Length != 0 && strArray.Length + 1 == c.Length)
									{
										for (int i = 0; i < c.Length; i++)
										{

											newscontent = new NewsContent();
											//string ctt = c[i].Replace("\n", "").Replace("\t", "").Replace("\r", "").Replace("；", "。").Replace(";", "。");
											string ctt = c[i].Replace(a1, "").Replace(b1, "").Replace(c1, "").Replace("\n", "").Replace(" ", "").Replace("\t", "").Replace("\r", "").Replace("；", "。").Replace(";", "。");
											string ct = string.Empty;
											string[] c12 = ctt.Split('。', StringSplitOptions.RemoveEmptyEntries);
											for (int h = 0; h < c12.Length; h++)
											{
												if (c12[h].Replace(" ", "").Length > 0)
												{
													ct += "<p>    " + c12[h] + "。</p><br>";
												}
											}
											ct += "<hr>";
											newscontent.Content = i == 0 ? c[i] : "<font>" + strArray[i - 1] + "</font><br>" + "    " + ct;
											newscontent.LawID = news.LawID;
											if (news.Title.Length > 200)
											{
												newscontent.PathName = news.Title.Substring(0, 200) + "=>" + (i == 0 ? "" : strArray[i - 1]);

											}
											newscontent.Orders = news.LawID + "-" + i.ToString().PadLeft(5, '0');
											newscontent.Type = context.Request.Properties["LawType"]?.Replace(" ", "");
											news.newscontents.Add(newscontent);
										}
									}
								}
							}
							context.AddData(typeName, news);

						}
						else
						{
							using (var conn = new SqlConnection(Options_.StorageConnectionString))
							{
								conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('参数有误" + CommonHelper.NoHtml(news.Title + context.Request.RequestUri.AbsoluteUri) + news.LawID+ news.Type  + ",  :detailParser,Parse' ,'" + DateTime.Now.ToString() + "') ");
							}
						}
					}
					else
					{
						using (var conn = new SqlConnection(Options_.StorageConnectionString))
						{
							conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(context.Request.RequestUri.AbsoluteUri) + news.LawID + news.Type + "','" + DateTime.Now.ToString() + "') ");
						}
					}
					return Task.CompletedTask;
				}
				catch (Exception ex)
				{
					using (var conn = new SqlConnection(Options_.StorageConnectionString))
					{
						conn.Execute($"INSERT INTO recordnoall ( [conditionText],[createtimes]) VALUES ('" + CommonHelper.NoHtml(ex.Message + ex.StackTrace + news.Title + context.Request.RequestUri.AbsoluteUri) + news.LawID + news.Type + ",  :detailParser,Parse' ,'" + DateTime.Now.ToString() + "') ");
					}
					return Task.CompletedTask;
				}

			}
		}

		protected class News
		{
			public string LawID { get; set; }
			public string Title { get; set; }
			public string Organ { get; set; }

			public string StartDate { get; set; }
			public string ImplementDate { get; set; }
			public string Type { get; set; }
			public string Valid { get; set; } = "有效";

			public string Other { get; set; }

			public List<NewsContent> newscontents = new List<NewsContent>();
			public string Detailpage { get; set; }
			public string ValidDate { get; set; }

			public string ImplementCode { get; set; }

			public string Path { get; set; }
			public string LawType { get; set; }

			

		}
		protected class NewsContent
		{
			public NewsContent()
			{
				ContentId = Guid.NewGuid().ToString();
			}
			public string ContentId { get; set; } 
			public string LawID { get; set; }
			public string PathName { get; set; }
			public string Content { get; set; }
			public string Orders { get; set; }
			public string Type { get; set; }

			public int times { get; set; } = 0;//引用次数

		}








	}
}
